# Regress essay content on income for Tables 2 and S1

library(caret)
library(tidyverse)

merged_df <- read.csv("merged_final.csv")
merged_df <- merged_df[merged_df$FAMILY_INCOME > 10000, ]
merged_df <- merged_df[!is.na(merged_df$FAMILY_INCOME),]

set.seed(1993)
merged_folds <- trainControl(method = "cv", number = 10)
merged_topics <- as.matrix(log(merged_df[,6:75]))

min_merged_cor <- abs(cor(merged_df$FAMILY_INCOME, 
                          log(merged_df[,6:75])))
# Drop min cor, topic 51

merged_topics <- merged_topics[,-51]
merged_df <- as.data.frame(cbind(merged_df$FAMILY_INCOME, merged_topics))

merged_mod <- train(V1 ~ ., method = "lm",
                    data = merged_df, trControl = merged_folds)

print(merged_mod)

################################################################################

creativ_df <- read.csv("creative_final.csv")
creativ_df <- creativ_df[creativ_df$FAMILY_INCOME > 10000, ]
creativ_df <- creativ_df[!is.na(creativ_df$FAMILY_INCOME),]

set.seed(1993)
creativ_folds <- trainControl(method = "cv", number = 10)
creativ_topics <- as.matrix(log(creativ_df[,6:55]))

min_creativ_cor <- abs(cor(creativ_df$FAMILY_INCOME, 
                           log(creativ_df[,6:55])))
# Drop min cor, topic 21

creativ_topics <- creativ_topics[,-21]

creativ_df <- as.data.frame(cbind(creativ_df$FAMILY_INCOME, creativ_topics))

creativ_mod <- train(V1 ~ ., method = "lm",
                     data = creativ_df, trControl = creativ_folds)

print(creativ_mod)

################################################################################

signif_df <- read.csv("signif_final.csv")
signif_df <- signif_df[signif_df$FAMILY_INCOME > 10000, ]
signif_df <- signif_df[!is.na(signif_df$FAMILY_INCOME),]

set.seed(1993)
signif_folds <- trainControl(method = "cv", number = 10)
signif_topics <- as.matrix(log(signif_df[,6:55]))

min_signif_cor <- abs(cor(signif_df$FAMILY_INCOME, 
                          log(signif_df[,6:55])))
# Drop min cor, topic 23

signif_topics <- signif_topics[,-23]

signif_df <- as.data.frame(cbind(signif_df$FAMILY_INCOME, signif_topics))

signif_mod <- train(V1 ~ ., method = "lm",
                    data = signif_df, trControl = signif_folds)

print(signif_mod)
